package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSONObject;
import com.google.gson.GsonBuilder;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.LoginDownloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.api.realize.AbstractUserCrawler;
import com.spider.moudle.constant.Constant;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.ErrorLogUtil;
import com.spider.moudle.util.SettingsUtil;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;

/**
 * Created by qiuhao on 2017/12/18.
 */
public class MaLaCrawler extends AbstractCrawler {

    /**
     * 这个是每个网站的个性化配置，是从settings.json里面读取到的(必须)
     */
    private Setting setting = SettingsUtil.getSettingBySite("MaLa");

    ArrayList<Cookie> cookies;

    private int friendPageCount = 10;  //最多爬取10页好友信息

    /**
     * 用户主页地址(非必须)
     */
    private String index = setting.getIndex();


    public MaLaCrawler() {
        ResponseEntity entity = null;
        try {
            entity = DownloaderFactory.buildDownloader().get("https://www.mala.cn/");
            cookies = Cookie.headersTranlateToCookie(entity.getHeaders());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    @Override
    public void generateTask() {
        LoginDownloader downloader = DownloaderFactory.buildLoginDownloader(cookies);
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                entity = downloader.get(index);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                Document document = entity.getDocument();
                Elements elements = document.select(".fl_g");
                if (elements.isEmpty()) {
                    return;
                }

                executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()),
                        new NamedThreadFactory(setting.getName(), "生成任务"));
                for (int i = 0; i < elements.size(); i++) {
                    Element element = elements.get(i);
                    Elements dtElements = element.select("dt");
                    if (dtElements.size() > 0) {
                        Element dtElement = dtElements.get(0);
                        String typeName = dtElement.select("a").get(0).text();  //版块名称
                        String url = dtElement.select("a").get(0).absUrl("href");
                        if (CommonUtil.isNotNull(typeName)) {
                            Area area = Area.search(typeName.replaceAll("论坛", ""));
                            MaLaCrawler.MaLaTaskCrawler default_task = new MaLaCrawler.MaLaTaskCrawler(url, area);
                            executorService.submit(default_task);
                        }
                    }
                }
            }
        } catch (Exception e) {
            //e.printStackTrace();
        } finally {
            if (null != executorService) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    /**
     * 内容爬取
     *
     * @param task
     * @return
     * @throws Exception
     */
    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        LoginDownloader downloader = DownloaderFactory.buildLoginDownloader(cookies);
        //爬取言论
        if (task.getTaskType() == TaskType.SPEECH) {
            JSONObject item = new JSONObject();
            SpeechBean speechBean = new SpeechBean();
            String url = task.getUrl();
            speechBean.setUrl(url);
            speechBean.setArea(task.getArea());
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url);
            } catch (Exception e) {
                ErrorLogUtil.error(e.getMessage(), e);
            }
            int page = 1;
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
                    Elements pagesElements = document.select(".plhin");
                    if (pagesElements != null && pagesElements.size() > 0) {
                        Element speechManElement = pagesElements.get(0);  //发帖人的所有信息
                        Elements lockeElements = speechManElement.select(".locked");
                        if (lockeElements == null || lockeElements.size() == 0) { //爬取未被禁言的帖子
                            Elements readAndReplyElements = document.select(".xi1");
                            int read_count = Integer.valueOf(readAndReplyElements.get(0).text());
//                            int reply_count = Integer.valueOf(readAndReplyElements.get(1).text());
                            speechBean.setRead_count(read_count);

                            Element titleElement = document.select(".ts").get(0);
                            String title = CommonUtil.handle(titleElement.select("#thread_subject").get(0).text());
                            speechBean.setTitle(title);
                            Date now = new Date();
                            speechBean.setSpider_time(now);
                            String publish_user = "";
                            String publish_user_photo = "";
                            String publish_time = "";
                            String publish_user_id = "";
                            String content = "";
                            String[] urlParam = url.split("-");
                            String publish_user_url = "";
                            // 帖子的Id
                            String id = urlParam[1];
                            speechBean.setId(id);
                            List<String> img_urls = new ArrayList<>();
                            Elements nameElements = speechManElement.select(".xw1");
                            if (nameElements != null && nameElements.size() > 0) {
                                publish_user = nameElements.get(0).text();   //发帖人名称
                                speechBean.setPublish_user(publish_user);
                                publish_user_url = nameElements.get(0).absUrl("href");
                                if (!publish_user_url.equals("javascript:;")) {
                                    publish_user_id = nameElements.get(0).absUrl("href").split("uid=")[1];
                                    speechBean.setPublish_user_id(publish_user_id);
                                }
                            }
                            Elements photoElements = speechManElement.select(".avatar"); //头像
                            if (photoElements != null && photoElements.size() > 0) {
                                Elements imgElements = photoElements.get(0).select("img");
                                if (imgElements != null && imgElements.size() > 0) {
                                    publish_user_photo = imgElements.get(0).attr("src");
                                    speechBean.setPublish_user_photo(publish_user_photo);
                                }
                            }
                            Elements authiElements = speechManElement.select(".authi");
                            if (authiElements != null && authiElements.size() > 0) {
                                switch (authiElements.size()) {   //发帖时间
                                    case 1:
                                        String[] params = authiElements.get(0).select("em").text().split(" ");
                                        publish_time = (DateUtil.str_date_str(params[1] + " " + params[2] + ":00", "yyyy-MM-dd HH:mm:ss"));
                                        speechBean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                                publish_time));
                                        break;
                                    case 2:
                                        String[] param1s = authiElements.get(1).select("em").text().split(" ");
                                        publish_time = (DateUtil.str_date_str(param1s[1] + " " + param1s[2] + ":00", "yyyy-MM-dd HH:mm:ss"));
                                        speechBean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                                publish_time));
                                        break;
                                    default:
                                        break;
                                }
                            }

                            Elements tipElements = speechManElement.select(".tip.tip_4");
                            if (!tipElements.isEmpty()) {
                                tipElements.remove();
                            }
                            Elements contentElements = speechManElement.select(".t_f");
                            if (contentElements != null && contentElements.size() > 0) {
                                contentElements.get(0).select("iframe").remove();
                                content = contentElements.get(0).text();  //内容
                                speechBean.setContent(content);
                                Elements imgElements = contentElements.get(0).select(".zoom");  //图片
                                if (imgElements != null && contentElements.size() > 0) {
                                    for (int i = 0; i < imgElements.size(); i++) {
                                        String img = imgElements.get(i).attr("zoomfile");
                                        if (img.equals("")) {
                                            img = imgElements.get(i).attr("src");
                                        }
                                        if (!img.contains("https:")) {
                                            img = "https://www.mala.cn/" + img;
                                        }
                                        img_urls.add(img);
                                    }
                                }
                                Elements uploadElements = speechManElement.select(".pattl");
                                if (uploadElements != null && uploadElements.size() > 0) {
                                    Elements imgElement = uploadElements.get(0).select("img");
                                    if (imgElement.size() > 0) {
                                        for (int i = 0; i < imgElement.size(); i++) {
                                            String img = imgElement.get(i).attr("zoomfile");
                                            if (img.equals("")) {
                                                img = imgElements.get(i).attr("src");
                                            }
                                            if (!img.contains("https:")) {
                                                img = "https://www.mala.cn/" + img;
                                            }
                                            img_urls.add(img);
                                        }
                                    }
                                }

                                speechBean.setImg_urls(img_urls);
                            }
                            speechBean.setPublish_user_jsonid(setting.getName() + "_" + publish_user_id);


                            //将发帖人添加到任务
                            if (CommonUtil.isNotNull(publish_user_id)) {
                                String userKey = setting.getName() + "_" + publish_user_id;
                                Task hometask = new Task(publish_user_url, TaskType.USER, WebType.FORLUM, userKey);
                                hometask.put("publish_user", publish_user);
                                hometask.put("publish_user_photo", publish_user_photo);
                                TaskManager.add(setting.getUser_task(), hometask);
                            }

                            //第一页回复爬取
                            List<SpeechBean> reply_nodes = new ArrayList<>();
                            if (pagesElements != null && pagesElements.size() > 1) {
                                for (int i = 1; i < pagesElements.size(); i++) {
                                    Elements lockedElements = pagesElements.get(i).select(".locked");
                                    if (lockedElements == null || lockedElements.size() == 0) {
                                        SpeechBean reply = new SpeechBean();
                                        String replyUser = "";
                                        String replyUserPhoto = "";
                                        String replyUserUrl = "";
                                        String replyUserId = "";
                                        String replyPublishTime = "";
                                        String replyContent = "";
                                        String replyId = "";
                                        String parentId = id;
                                        List<String> reply_img_urls = new ArrayList<>();
                                        Element pagesElement = pagesElements.get(i);
                                        Elements replytipsElements = pagesElement.select(".tip.tip_4");
                                        if (!replytipsElements.isEmpty()) {
                                            replytipsElements.remove();
                                        }
                                        String replyUrl = url + "#" + pagesElement.id();
                                        Elements replyNameElements = pagesElement.select(".xw1");
                                        if (replyNameElements != null && replyNameElements.size() > 0) {
                                            replyUser = replyNameElements.get(0).text();
                                            replyUserUrl = replyNameElements.get(0).absUrl("href");
                                            if (!replyUserUrl.equals("javascript:;")) {
                                                replyUserId = replyNameElements.get(0).absUrl("href").split("uid=")[1];
                                            }
                                        }
                                        Elements avatarElements = pagesElement.select(".avatar");
                                        if (avatarElements != null && avatarElements.size() > 0) {
                                            replyUserPhoto = avatarElements.get(0).select("img").get(0).attr("src");
                                        }
                                        pagesElement.select(".tip.tip_4").remove();
                                        Elements tfElements = pagesElement.select(".t_f");
                                        if (tfElements.size() > 0) {
                                            Elements quoteElements = tfElements.get(0).select(".quote");
                                            if (quoteElements != null && quoteElements.size() > 0) {  //回复的回复
                                                Elements aElements = quoteElements.get(0).select("a");
                                                if (aElements != null && aElements.size() > 0) {
                                                    String href = aElements.get(0).attr("href");
                                                    String[] params = href.split("&pid=");
                                                    parentId = params[1].split("&")[0];
                                                }
                                                quoteElements.remove();
                                                replyContent = tfElements.get(0).text();
                                                Elements replyImgElements = tfElements.get(0).select("img");
                                                if (replyImgElements != null && replyImgElements.size() > 0) {
                                                    for (int replyindex = 0; replyindex < replyImgElements.size(); replyindex++) {
                                                        String img = replyImgElements.get(replyindex).attr("zoomfile");
                                                        if (img.equals("")) {
                                                            img = replyImgElements.get(replyindex).attr("src");
                                                        }
                                                        if (!img.contains("https:")) {
                                                            img = "https://www.mala.cn/" + img;
                                                        }
                                                        reply_img_urls.add(img);
                                                    }
                                                }


                                            } else {
                                                replyContent = pagesElement.select(".t_f").text();
                                                Elements replyImgElements = pagesElement.select(".t_f").select("img");
                                                if (replyImgElements != null && replyImgElements.size() > 0) {
                                                    for (int replyindex = 0; replyindex < replyImgElements.size(); replyindex++) {
                                                        String img = replyImgElements.get(replyindex).attr("zoomfile");
                                                        if (img.equals("")) {
                                                            img = replyImgElements.get(replyindex).attr("src");
                                                        }
                                                        if (!img.contains("https:")) {
                                                            img = "https://www.mala.cn/" + img;
                                                        }
                                                        reply_img_urls.add(img);
                                                    }
                                                }

                                            }
                                            replyId = pagesElement.select(".t_f").get(0).attr("id").split("_")[1];
                                        }

                                        Elements uploadElements = pagesElement.select(".pattl");
                                        if (uploadElements != null && uploadElements.size() > 0) {
                                            uploadElements.get(0).select(".tip.tip_4").remove();
                                            Elements imgElement = uploadElements.get(0).select("img");
                                            if (imgElement.size() > 0) {
                                                for (int j = 0; j < imgElement.size(); j++) {
                                                    String img = imgElement.get(j).attr("zoomfile");
                                                    reply_img_urls.add(img);
                                                }
                                            }
                                        }


                                        Elements authElements = pagesElement.select(".authi");
                                        if (authElements != null && authElements.size() > 0) {
                                            switch (authElements.size()) {   //发帖时间
                                                case 1:
                                                    String[] params = authElements.get(0).select("em").text().split(" ");
                                                    replyPublishTime = (DateUtil.str_date_str(params[1] + " " + params[2] + ":00", "yyyy-MM-dd HH:mm:ss"));
                                                    break;
                                                case 2:
                                                    String[] param1s = authElements.get(1).select("em").text().split(" ");
                                                    replyPublishTime = (DateUtil.str_date_str(param1s[1] + " " + param1s[2] + ":00", "yyyy-MM-dd HH:mm:ss"));
                                                    break;
                                                default:
                                                    break;
                                            }
                                        }
                                        if (CommonUtil.isNotNull(replyUserId)) {
                                            String userKey = setting.getName() + "_" + replyUserId;
                                            Task hometask = new Task(replyUserUrl, TaskType.USER, WebType.FORLUM, userKey);
                                            hometask.put("publish_user", replyUser);
                                            hometask.put("publish_user_photo", replyUserPhoto);
                                            TaskManager.add(setting.getUser_task(), hometask);
                                        }
                                        reply.setPublish_user(replyUser);
                                        reply.setPublish_user_id(replyUserId);
                                        reply.setPublish_user_jsonid(setting.getName() + "_" + replyUserId);
                                        reply.setPublish_user_photo(replyUserPhoto);
                                        reply.setContent(replyContent.trim());
                                        reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, replyPublishTime));
                                        reply.setUrl(replyUrl);
                                        reply.setId(replyId);
                                        reply.setParent_id(parentId);
                                        reply.setAncestor_id(id);
                                        reply.setImg_urls(reply_img_urls);
                                        reply.setArea(task.getArea());
                                        reply_nodes.add(reply);
                                    }
                                }
                            }

                            //后面几页回复爬取
                            Elements pageElements = document.select(".pgs.mtm.mbm.cl").get(0).select(".pg");
                            if (pageElements.size() > 0) {
                                Elements pElements = pageElements.select("a");
                                if (pElements != null && pElements.size() > 1) {
                                    page = Integer.valueOf(pageElements.get(0).select("label").get(0).select("span").get(0).attr("title").split(" ")[1]);
                                }
                            }
                            //爬虫最大页码的限制，超过最大页的帖子就只抓到最大页码数
                            if (page >= setting.getMax_page_count()) {
                                page = setting.getMax_page_count();
                            }
                            if (page > 1) {
                                int threadNum = setting.getReply_thread_count();
                                List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                                ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, page, threadNum);
                                ExecutorService executorService = null;
                                try {
                                    executorService = ExecutorServiceFactoy.buildExecutorService(splitPages.size(), ExecutorServiceFactoy.getName(setting.getName(), url),
                                            new NamedThreadFactory(setting.getName(), "回复"));
                                    for (int i = 0; i < splitPages.size(); i++) {
                                        MaLaCrawler.ReplyCrawler callable = new MaLaCrawler.ReplyCrawler(splitPages.get(i), url, id, task.getArea());
                                        FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                } finally {
                                    if (null != executorService) {
                                        executorService.shutdown();
                                    }
                                }
                                for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                                    reply_nodes.addAll(futureTask.get());
                                }

                                Collections.sort(reply_nodes, (o1, o2) -> {
                                    Date s1 = o1.getPublish_time();
                                    Date s2 = o2.getPublish_time();
                                    int flag = s1.compareTo(s2);
                                    return flag;
                                });
                            }
                            speechBean.setReply_nodes(reply_nodes);
                            speechBean.setReply_count(reply_nodes.size());
                            item.put("data", speechBean.toJSONObject());
                            return item;
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    ErrorLogUtil.error("麻辣社区获取言论数据解析错误: " + url);
                }

            }
        } else if (task.getTaskType() == TaskType.USER)

        {  //爬取用户主页
            String userKey = task.getTaskKey();
            if (CommonUtil.isNull(userKey)) {
                System.err.println("User key is not empty where task type belong to home page");
                return null;
            }
            UserBean data;
            JSONObject item = new JSONObject();
            String url = task.getUrl();
            try {
                data = new UserBean();
                data.setJson_id(userKey);
                data.setUrl(url);
                String publish_user_id = userKey.substring(userKey.indexOf("_") + 1);
                data.setPublish_user_id(publish_user_id);
                String publish_user = (String) task.get("publish_user");
                data.setPublish_user(publish_user);
                String publish_user_photo = (String) task.get("publish_user_photo");
                data.setPublish_user_photo(publish_user_photo);
                String userHomeUrl = url + "&do=profile";  //用户详细信息url
                int article_count = 0;  //发帖数
                int reply_article_count = 0; //回复数
                int friend_count = 0; //好友数
                int visit_count = 0; //主页访问量
                String sex = "";
                String group_name = "";

                //好友(有好友就需要爬好友信息)
                List<UserBean> friend_details = new ArrayList<>();
                //粉丝(有粉丝就需要爬粉丝信息)
                List<UserBean> fans_details = new ArrayList<>();
                //关注(有关注用户就需要爬关注用户信息)
                List<UserBean> follow_details = new ArrayList<>();
                ResponseEntity user_home_entity = null;
                try {
                    user_home_entity = downloader.get(userHomeUrl);
                } catch (Exception e) {
                    ErrorLogUtil.error(setting.getName() + "访问用户详细信息主页：" + userHomeUrl + "出错", e);
                }
                if (user_home_entity != null && user_home_entity.getStatus_code() == 200) {
                    Document document = user_home_entity.getDocument();
                    Elements countInfoElements = document.select("ul.cl.bbda.pbm.mbm");  //用户的统计信息
                    String friend_url = "";   //好友数量页面url
                    if (countInfoElements != null && countInfoElements.size() > 0) {
                        Element countInfoElement = countInfoElements.get(0);
                        Elements aElements = countInfoElement.select("a");
                        friend_url = aElements.get(0).absUrl("href");

                        String friendCount[] = aElements.get(0).text().split(" ");
                        if (friendCount.length == 2) {
                            friend_count = Integer.valueOf(friendCount[1]);
                        }

                        reply_article_count = Integer.valueOf(aElements.get(4).text().split(" ")[1]);

                        String articleCount[] = aElements.get(5).text().split(" ");
                        if (articleCount.length == 2) {
                            article_count = Integer.valueOf(articleCount[1]);
                        }


                    }
                    Elements pbmElements = document.select(".pbm.mbm.bbda.cl");
                    if (pbmElements != null && pbmElements.size() > 0) {
                        Element ulElement = pbmElements.get(0).select("ul").get(1);
                        Elements introducElements = ulElement.select("li");
                        if (introducElements != null && introducElements.size() > 0) {
                            for (int i = 0; i < introducElements.size(); i++) {
                                String text = CommonUtil.trim(introducElements.get(i).select("em").get(0).text());
                                if (text.equals("个人签名")) {
                                    String introduction = introducElements.get(i).select("table").get(0).text();
                                    data.setIntroduction(introduction);
                                    break;
                                }
                            }
                        }
                    }
                    Elements xi1Elements = document.select(".xi1");
                    if (xi1Elements != null && xi1Elements.size() > 0) {
                        if (!xi1Elements.get(0).text().equals("")) {
                            visit_count = Integer.valueOf(xi1Elements.get(0).text());
                        }
                    }
                    Elements userBasicInfoElements = document.select(".pf_l.cl");  //用户基本信息
                    if (userBasicInfoElements.size() > 1) {
                        Elements basicElements = userBasicInfoElements.get(1).select("li");
                        if (basicElements != null && basicElements.size() > 0) {
                            String tag = basicElements.get(0).select("em").text();
                            if (tag.equals("真实姓名")) {
                                userBasicInfoElements.get(1).select("li").get(1).select("em").get(0).remove();
                                sex = CommonUtil.trim(userBasicInfoElements.get(1).select("li").get(1).text().replaceAll("\"", ""));
                            } else if (tag.equals("性别")) {
                                userBasicInfoElements.get(1).select("li").get(0).select("em").get(0).remove();
                                sex = CommonUtil.trim(userBasicInfoElements.get(1).select("li").get(0).text().replaceAll("\"", ""));
                            }
                        }
                    }

                    Elements groupnameElements = null;
                    try {
//                        groupnameElements = pbmElements.get(3).select("ul").get(0).select("li");
                        groupnameElements = document.select(".xg1~span");
                    } catch (Exception e) {
                        System.out.println("获取用户组出错："+url);
                        e.printStackTrace();
                    }
                    if (groupnameElements.size() == 1) {
                        group_name = groupnameElements.get(0).select("a").text();
                    } else if (groupnameElements.size() == 2) {
                        group_name = groupnameElements.get(1).select("a").text();
                    }
                    String register_time = "";
                    Elements pbbsElements = document.select("#pbbs");
                    if (pbbsElements.size() > 0) {
                        String tag = pbbsElements.get(0).select("li").get(0).select("em").get(0).text();
                        if (tag.equals("在线时间")) {
                            pbbsElements.get(0).select("li").get(1).select("em").remove();
                            register_time = DateUtil.str_date_str(pbbsElements.get(0).select("li").get(1).text().replaceAll("\"", "") + ":00", DateUtil.yyyy_MM_dd_HH_mm_ss);
                        } else {
                            pbbsElements.get(0).select("li").get(0).select("em").remove();
                            register_time = DateUtil.str_date_str(pbbsElements.get(0).select("li").get(0).text().replaceAll("\"", "") + ":00", DateUtil.yyyy_MM_dd_HH_mm_ss);
                        }
                    }
                    if (friend_count > 0) {
                        ResponseEntity friendEntity = downloader.get(friend_url);
                        if (friendEntity != null && friendEntity.getStatus_code() == 200) {
                            Document frientDocument = friendEntity.getDocument();
                            Elements friendElements = frientDocument.select("li.bbda.cl");
                            if (friendElements.size() > 0) {
                                for (int i = 0; i < friendElements.size(); i++) {
                                    UserBean friendBean = new UserBean();
                                    Element aElement = friendElements.get(i).select("h4").select("a").get(0);
                                    String friendUrl = aElement.absUrl("href");
                                    String friendName = aElement.text();
                                    String friendId = aElement.attr("href").split("uid=")[1];
                                    String friendPhoto = friendElements.get(i).select(".avt").get(0).select("a").get(0).select("img").get(0).absUrl("src").replaceAll("small", "middle");
                                    String friendUserkey = setting.getName() + "_" + friendId;
                                    friendBean.setPublish_user(friendName);
                                    friendBean.setPublish_user_photo(friendPhoto);
                                    friendBean.setPublish_user_id(friendId);
                                    friendBean.setUrl(friendUrl);
                                    friendBean.setJson_id(friendUserkey);
                                    friend_details.add(friendBean);
                                }
                            }
                            //分页的爬取好友的信息
                            Elements pgElements = frientDocument.select(".pg");
                            if (pgElements != null && pgElements.size() > 0) {
                                Elements labelElements = pgElements.get(0).select("label");
                                if (labelElements != null && labelElements.size() > 0) {
                                    int pagenum = Integer.valueOf(labelElements.get(0).select("span").get(0).attr("title").split(" ")[1]);
                                    if (pagenum > friendPageCount) {
                                        pagenum = friendPageCount;
                                    }
                                    int threadNum = 1;
                                    if (pagenum > 4) {
                                        threadNum = pagenum / 2;
                                    }
                                    ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, pagenum, threadNum);
                                    List<FutureTask<List<UserBean>>> futureTasks = new ArrayList<>();
                                    ExecutorService executorService = null;
                                    try {
                                        executorService = ExecutorServiceFactoy.buildExecutorService(splitPages.size(), ExecutorServiceFactoy.getName(setting.getName(), friend_url),
                                                new NamedThreadFactory(setting.getName(), "用户好友"));
                                        for (int i = 0; i < splitPages.size(); i++) {
                                            MaLaCrawler.FriendUserCramler callable = new MaLaCrawler.FriendUserCramler(splitPages.get(i), friend_url, setting.getName());
                                            FutureTask<List<UserBean>> futureTask = new FutureTask<>(callable);
                                            futureTasks.add(futureTask);
                                            executorService.submit(futureTask);
                                        }
                                        if (futureTasks.size() > 0) {
                                            for (FutureTask<List<UserBean>> futureTask : futureTasks) {
                                                friend_details.addAll(futureTask.get());
                                            }
                                        }
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                    } finally {
                                        if (null != executorService) {
                                            executorService.shutdown();
                                        }
                                    }
                                }
                            }
                        }
                    }
                    data.setArticle_count(article_count);
                    data.setSex(sex);
                    data.setVisit_count(visit_count);
                    data.setGroup_name(group_name);
                    data.setReply_article_count(reply_article_count);
                    try {
                        if(register_time.equals("")){
                            return null;
                        }
                        data.setRegister_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, register_time));
                    } catch (Exception e) {
                        System.out.println("获取注册时间出错:--->" + url);
                        e.printStackTrace();
                    }
                    data.setFriend_details(friend_details);
                    data.setFriend_count(friend_count);
                    data.setFans_details(fans_details);
                    data.setFans_count(fans_details.size());
                    data.setFollow_details(follow_details);
                    data.setFollow_count(follow_details.size());
                    item.put("data", data.toJSONObject());
                }
                return item;
            } catch (Exception e) {
                e.printStackTrace();
                ErrorLogUtil.error(e.getMessage(), e);
            }
        }
        return null;
    }

    /**
     * 爬取用户的好友
     */
    public class FriendUserCramler extends AbstractUserCrawler {

        private ArrayList<Integer> pages;

        private String settingName;

        public FriendUserCramler(ArrayList<Integer> pages, String url, String settingName) {
            this.pages = pages;
            this.url = url;
            this.settingName = settingName;

        }

        @Override
        public List<UserBean> call() throws Exception {
            LoginDownloader downloader = DownloaderFactory.buildLoginDownloader(cookies);
            List<UserBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                String nextUrl = url + "&page=" + pages.get(k);
                try {
                    ResponseEntity friendEntity = downloader.get(nextUrl);
                    if (friendEntity != null && friendEntity.getStatus_code() == 200) {
                        Document frientDocument = friendEntity.getDocument();
                        Elements friendElements = frientDocument.select("li.bbda.cl");
                        if (friendElements.size() > 0) {
                            for (int i = 0; i < friendElements.size(); i++) {
                                UserBean friendBean = new UserBean();
                                Element aElement = friendElements.get(i).select("h4").select("a").get(0);
                                String friendUrl = aElement.absUrl("href");
                                String friendName = aElement.text();
                                String friendId = aElement.attr("href").split("uid=")[1];
                                String friendPhoto = friendElements.get(i).select(".avt").get(0).select("a").get(0).select("img").get(0).absUrl("src").replaceAll("small", "middle");
                                String userKey = settingName + "_" + friendId;
                                friendBean.setPublish_user(friendName);
                                friendBean.setPublish_user_photo(friendPhoto);
                                friendBean.setPublish_user_id(friendId);
                                friendBean.setUrl(friendUrl);
                                friendBean.setJson_id(userKey);
                                array.add(friendBean);
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    ErrorLogUtil.error("麻辣社区用户好友爬取: " + nextUrl + "出错", e);
                }
            }
            return array;
        }
    }


    /**
     * 抓回复的线程(这个是个性化的每个网站都不一样)
     */
    public class ReplyCrawler extends AbstractReplyCrawler {

        private ArrayList<Integer> pages;

        public ReplyCrawler(ArrayList<Integer> pages, String url, String parentId, Area area) {
            this.pages = pages;
            this.url = url;
            this.parentId = parentId;
            this.area = area;
        }

        @Override
        public List<SpeechBean> call() throws Exception {
            LoginDownloader downloader = DownloaderFactory.buildLoginDownloader(cookies);
            List<SpeechBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                String urlParams[] = url.split("-");
                String nextUrl = urlParams[0] + "-" + urlParams[1] + "-" + pages.get(k) + "-" + urlParams[3];
                try {
                    Document document = null;
                    ResponseEntity entity = downloader.get(nextUrl);
                    if (entity.getStatus_code() == 200) {
                        document = entity.getDocument();
                        Elements pagesElements = document.select(".plhin");
                        if (pagesElements != null && pagesElements.size() > 1) {
                            for (int i = 0; i < pagesElements.size(); i++) {
                                SpeechBean reply = new SpeechBean();
                                String replyUser = "";
                                String replyUserPhoto = "";
                                String replyUserUrl = "";
                                String replyUserId = "";
                                String replyPublishTime = "";
                                String replyContent = "";
                                String replyId = "";
                                String ancestorId = parentId;
                                List<String> reply_img_urls = new ArrayList<>();
                                Element pagesElement = pagesElements.get(i);
                                Elements replytipsElements = pagesElement.select(".tip.tip_4");
                                if (!replytipsElements.isEmpty()) {
                                    replytipsElements.remove();
                                }
                                String replyUrl = nextUrl + "#" + pagesElement.id();
                                Elements replyNameElements = pagesElement.select(".xw1");
                                if (replyNameElements != null && replyNameElements.size() > 0) {
                                    replyUser = replyNameElements.get(0).text();
                                    replyUserUrl = replyNameElements.get(0).absUrl("href");
                                    if (!replyUserUrl.equals("javascript:;")) {
                                        replyUserId = replyNameElements.get(0).absUrl("href").split("uid=")[1];
                                    }
                                }
                                Elements avatarElements = pagesElement.select(".avatar");
                                if (avatarElements != null && avatarElements.size() > 0) {
                                    replyUserPhoto = avatarElements.get(0).select("img").get(0).absUrl("src");
                                }
                                Elements tfElements = pagesElement.select(".t_f");
                                if (tfElements.size() > 0) {
                                    Elements quoteElements = tfElements.get(0).select(".quote");
                                    if (quoteElements != null && quoteElements.size() > 0) {  //回复的回复
                                        Elements aElements = quoteElements.get(0).select("a");
                                        if (aElements != null && aElements.size() > 0) {
                                            String href = aElements.get(0).attr("href");
                                            String[] params = href.split("&pid=");
                                            parentId = params[1].split("&")[0];
                                        }
                                        quoteElements.remove();
                                        replyContent = tfElements.get(0).text();
                                        Elements replyImgElements = tfElements.get(0).select("img");
                                        if (replyImgElements != null && replyImgElements.size() > 0) {
                                            for (int replyindex = 0; replyindex < replyImgElements.size(); replyindex++) {
                                                String img = replyImgElements.get(replyindex).attr("zoomfile");
                                                if (img.equals("")) {
                                                    img = replyImgElements.get(replyindex).attr("src");
                                                }
                                                if (!img.contains("https:")) {
                                                    img = "https://www.mala.cn/" + img;
                                                }
                                                reply_img_urls.add(img);
                                            }
                                        }
                                    } else {
                                        replyContent = pagesElement.select(".t_f").text();
                                        Elements replyImgElements = pagesElement.select(".t_f").select("img");
                                        if (replyImgElements != null && replyImgElements.size() > 0) {
                                            for (int replyindex = 0; replyindex < replyImgElements.size(); replyindex++) {
                                                String img = replyImgElements.get(replyindex).attr("zoomfile");
                                                if (img.equals("")) {
                                                    img = replyImgElements.get(replyindex).attr("src");
                                                }
                                                if (!img.contains("https:")) {
                                                    img = "https://www.mala.cn/" + img;
                                                }
                                                reply_img_urls.add(img);
                                            }
                                        }

                                    }
                                    replyId = pagesElement.select(".t_f").get(0).attr("id").split("_")[1];
                                }
                                Elements authiElements = pagesElement.select(".authi");
                                if (authiElements != null && authiElements.size() > 0) {
                                    switch (authiElements.size()) {   //发帖时间
                                        case 1:
                                            String[] params = authiElements.get(0).select("em").text().split(" ");
                                            replyPublishTime = (DateUtil.str_date_str(params[1] + " " + params[2] + ":00", DateUtil.yyyy_MM_dd_HH_mm_ss));
                                            break;
                                        case 2:
                                            String[] param1s = authiElements.get(1).select("em").text().split(" ");
                                            replyPublishTime = (DateUtil.str_date_str(param1s[1] + " " + param1s[2] + ":00", DateUtil.yyyy_MM_dd_HH_mm_ss));
                                            break;
                                        default:
                                            break;
                                    }
                                }
                                if (CommonUtil.isNotNull(replyUserId)) {
                                    String userKey = setting.getName() + "_" + replyUserId;
                                    Task hometask = new Task(replyUserUrl, TaskType.USER, WebType.FORLUM, userKey);
                                    hometask.put("publish_user", replyUser);
                                    hometask.put("publish_user_photo", replyUserPhoto);
                                    TaskManager.add(setting.getUser_task(), hometask);
                                }
                                reply.setPublish_user(replyUser);
                                reply.setPublish_user_id(replyUserId);
                                reply.setPublish_user_jsonid(setting.getName() + "_" + replyUserId);
                                reply.setPublish_user_photo(replyUserPhoto);
                                reply.setContent(replyContent);
                                reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, replyPublishTime));
                                reply.setUrl(replyUrl);
                                reply.setId(replyId);
                                reply.setParent_id(parentId);
                                reply.setAncestor_id(ancestorId);
                                reply.setImg_urls(reply_img_urls);
                                reply.setArea(area);
                                array.add(reply);
                            }
                        }
                    }
                } catch (Exception e) {
                    ErrorLogUtil.error("麻辣社区帖子回复: " + nextUrl + "出错", e);
                }
            }
            return array;
        }
    }

    /**
     * 任务爬取
     */
    public class MaLaTaskCrawler implements Runnable {

        LoginDownloader downloader = DownloaderFactory.buildLoginDownloader(cookies);

        private String url;

        private Area area = null;

        private String hotUrl;
        int page = 1;

        public MaLaTaskCrawler(String url, Area area) {
            this.url = url;
            this.area = area;
        }

        @Override
        public void run() {

            handle();
            hot();
        }


        private void handle() {

            String lastUrl = "";
            try {
                ResponseEntity entity = null;
                try {
                    entity = downloader.get(url);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
                if (entity != null && entity.getStatus_code() == 200) {
                    Document doc = entity.getDocument();
                    Elements pgElements = doc.select(".pg");
                    if (pgElements.size() > 0) {
                        Elements aElements = pgElements.get(0).select("a");
                        if (aElements.size() > 4) {
                            page = 5;
                        } else {
                            page = aElements.size();   //获得总页数
                        }
                    }
                    Elements tfElements = doc.select(".tf");
                    if (tfElements.size() > 0) {
                        Elements xi2Elements = tfElements.get(0).select(".xi2");
                        if (xi2Elements != null && xi2Elements.size() > 1) {
                            lastUrl = xi2Elements.get(1).absUrl("href");
                        }
                        if (xi2Elements != null && xi2Elements.size() > 2) {
                            hotUrl = xi2Elements.get(2).absUrl("href");
                        }
                    }
                    if (lastUrl.equals("")) {
                        lastUrl = url;  //最新的Url
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
            for (int p = 0; p < page; p++) {
                boolean flag = false;
                try {
                    ResponseEntity entityy = null;
                    try {
                        entityy = downloader.get(lastUrl);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entityy != null && entityy.getStatus_code() == 200) {
                        Document document = entityy.getDocument();
                        Elements elements = document.select("[id^=normalthread_]");
                        for (int i = 0; i < elements.size(); i++) {
                            Element element = elements.get(i);
                            Elements a_list = element.select("a");
                            if (CommonUtil.isNull(a_list)) {
                                continue;
                            }
                            String speech_url = a_list.get(0).absUrl("href");
                            if (CommonUtil.isNotNull(speech_url)) {
                                Elements timeElements = element.select(".by");
                                String updateTimeString = null;
                                if (timeElements.size() > 1) {
                                    updateTimeString = timeElements.get(1).select("em").select("a").text() + ":00";
                                }
                                if (CommonUtil.isNotNull(updateTimeString)) {
                                    Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, updateTimeString);
                                    String speechkey = setting.getSite() + "_" + speech_url.split("-")[1].trim();
                                    Task task = new Task(speech_url, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                                    TaskManager.add(setting.getSpeech_task(), task);
                                    if (i == elements.size() - 1) {
                                        Date now = new Date();
                                        if (updateTime.getTime() < now.getTime() - Constant.LAST_48_HOUR_TIME_STAMP) {
                                            break;
                                        }
                                        if (page > 1) {   //不止一页
                                            if (document.select(".nxt").size() > 1) {
                                                String nextUrl = document.select(".nxt").get(1).absUrl("href");
                                                lastUrl = nextUrl;
                                                flag = true;
                                            }
                                        }
                                    }
                                }

                            }

                        }

                    }
                    //不存在下一页退出循环
                    if (!flag) {
                        break;
                    }
                } catch (Exception e) {
                    log.error("麻辣社区生成最新任务出错: ", e);
                }
            }
        }

        private void hot() {  //热门帖子
            for (int p = 0; p < (page > 3 ? 3 : page); p++) {
                try {
                    ResponseEntity entityy = null;
                    try {
                        entityy = downloader.get(hotUrl);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entityy != null && entityy.getStatus_code() == 200) {
                        Document document = entityy.getDocument();
                        Elements elements = document.select("[id^=normalthread_]");
                        for (int i = 0; i < elements.size(); i++) {
                            Element element = elements.get(i);
                            Elements a_list = element.select("a");
                            if (CommonUtil.isNull(a_list)) {
                                continue;
                            }
                            String speech_url = a_list.get(0).absUrl("href");
                            if (CommonUtil.isNotNull(speech_url)) {
                                String speechkey = setting.getSite() + "_" + speech_url.split("-")[1].trim();
                                Task task = new Task(speech_url, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                                TaskManager.add(setting.getSpeech_task(), task);
                            }
                        }
                        if (page > 1) {   //不止一页
                            if (document.select(".nxt").size() > 1) {
                                String nextUrl = document.select(".nxt").get(1).absUrl("href");
                                hotUrl = nextUrl;
                            }
                        }
                    }
                } catch (Exception e) {
                    log.error("麻辣社区生成热门任务出错: ", e);
                }
            }
        }
    }
}
